# -*- coding: utf-8 -*-

import os
import sys

if __name__ == "__main__":
    currentdir = os.path.dirname( os.path.abspath( sys.argv[0] ) )
    basedir = os.path.abspath( os.path.join( currentdir, ".." ) )
    pythondir = basedir
    sys.path.insert( 0, pythondir )
    pass

import traceback
import random
from time import time
from subprocess import Popen
from time import sleep
from datetime import timedelta
from multiprocessing import cpu_count

from logger import get_file_logger
from picklerpc.msghandler import RequestHandler
from picklerpc.sendmsg import sendmsg
from picklerpc import Server
from service import default_request_timeout, default_niceness_level, Py27ServiceManager as ServiceManager

__all__ = [
    'SearchDupImageVectorHandler',
    'SearchDupImageVectorServer',
    'SearchDupImageVectorRequest',
    'SearchDupImageVectorManager'
]

#---------------------------------------------

class SearchDupImageVectorHandler(RequestHandler):
    """
    Do some tasks, maybe in subprocess
    """
    
    def _request_handle(self, data):
        """
        @param data: simple object data
        @type: dict
        
        @return: python simple object data, jsonable via json.dumps
        @rtype: mixed
        """
        result = False
        
        if not (type(data) is dict):
            return result

        if data["task"] == "ping":
            return self.server.ping()
        if data["task"] == "quit":
            return self.server.quit()
        if data["task"] == "reset-stats":
            return self.server.reset_stats()
        if data["task"] == "empty":
            return self.server.empty()
        if data["task"] == "get-stats":
            return self.server.get_stats()
        if data["task"] == "process":
            data = data["data"]
            return self.server.process(data["row"])
        return result
    
    pass

#---------------------------------------------

class SearchDupImageVectorServer(Server):

    max_diff = 95.0
    min_diff = 5.0
    max_vector = 100.0

    iMage = None
    queue = None
    stop_event = None
    task_service_address = None
    _worker_thread = None

    count_items = 0
    count_processed = 0

    stats_req = None
    db_req = None

    _lock = None

    log = None

    def server_activate(self):
        try:
            # Python 3
            from queue import Queue
        except ImportError:
            # Python 2
            from Queue import Queue

        from threading import Thread, Event, Lock

        self.log = get_file_logger(self.__class__.__name__ + "-%s" % self.server_address[1])

        try:

            from libc import set_affinity

            set_affinity(0, 2**(self.server_address[1] % cpu_count()))

        except Exception as e:
            self.log.error(e)
            self.log.error(traceback.format_exc())

        self._lock = Lock()

        self.queue = Queue()
        self.stop_event = Event()
        self._worker_thread = Thread(target=self._worker)
        self._worker_thread.start()

        return Server.server_activate(self)

    def ping(self):
        return "pong"

    def find_dupimage_vectors(self, item):

        item_v10 = self.db_req.get_image_vectors10(item["id"])
        # self.log.debug("Item vector 10=%r" % item_v10)

        gray_ids = {}

        #self.log.debug("-- Get images gray like...")
        #t1 = datetime.now()

        for v10 in item_v10:
            imv = self.db_req.get_similar_images10(v10)

            for row in imv:
                gray_ids[ row['image_id'] ] = 1

        gray_ids = tuple(gray_ids.keys())
        gray_count = len(gray_ids)

        #t2 = datetime.now()
        #self.log.debug("-- Fetch images gray like: %s, in %s" % (gray_count, t2 - t1))

        item_vs_big = None
        # self.log.debug("Item vector 100=%r" % item_v100)

        if not gray_count:
            self.stats_req.inc("search_dupimages_processed", 1)

        for img_id in gray_ids:

            if self.stop_event.isSet():
                break

            self.stats_req.inc("search_dupimages_processed", 1.0 / gray_count)

            # self.log.debug("Item gray id=%r" % img_id)

            if img_id == item["id"]:
                #self.log.debug("-- Same image...")
                continue

            # self.log.debug("Item gray compare with item...")
            if self.db_req.has_image2image(item["id"], img_id):
                #self.log.debug("-- Already compared...")
                continue

            img = self.db_req.get_image(img_id)

            if not img:
                #self.log.debug("-- Image not found by id! Deleted?")
                continue

            if img["file_id"] == item["file_id"]:
                #self.log.debug("-- In the same file...")
                continue

            # self.log.debug("Item gray get vectors 100")
            img_vs_big = self.db_req.get_image_vectors_big(img_id)

            if not item_vs_big:
                item_vs_big = self.db_req.get_image_vectors_big(item["id"])

            # self.log.debug("Item gray vectors 100=%r" % img_v100)

            sim = 0
            comp = (0,)

            if len(img_vs_big) and len(item_vs_big):
                for item_v in item_vs_big:

                    if self.stop_event.isSet():
                        break

                    vbig_1 = item_v["vector"]

                    for img_v in img_vs_big:

                        if self.stop_event.isSet():
                            break

                        vbig_2 = img_v["vector"]

                        #comp += ( compare_vectors(v100_1, v100_2, min_diff, parts=int(self.max_vector)), )
                        comp += ( self.iMage.compare_vectors(
                            vbig_1, vbig_2,
                            min_diff=self.min_diff,
                            parts=int(self.max_vector)
                        ), )
                        # self.log.debug("Item gray compared vs 100=%r" % (comp,))
                        if max(comp) >= self.max_diff:
                            sim += 1
                            break

                    if sim:
                        break

            q_point = 0

            if sim:

                self.stats_req.inc("search_dupimages_similar", 1)

                #self.log.debug("-- found similar item=%r" % img)

                if item["mime"] != img["mime"]:
                    if item["mime"] == "image/png":
                        q_point += 2
                    if img["mime"] == "image/png":
                        q_point -= 2
                else:
                    if int(item["size"]) * 0.7 > int(img["size"]):
                        q_point += 1
                    if int(item["size"]) < int(img["size"]) * 0.7:
                        q_point -= 1

                if int(item["width"]) * int(item["height"]) > int(img["width"]) * int(img["height"]):
                    q_point += 2
                else:
                    q_point -= 2

                if int(item["is_gray"]) > int(img["is_gray"]):
                    q_point -= 3
                else:
                    q_point += 3

                #self.log.debug("-- equal by %s, quality points: %s" % (max(comp), q_point,))
                self.db_req.insert_image2image(
                    item["id"], img_id,
                    item["file_id"], img["file_id"],
                    max(comp), q_point)

                file2file = self.db_req.has_file2file(item["file_id"], img["file_id"] )
                #self.log.debug("-- has File2File: %r" % (file2file,))
                if not file2file:

                    file1_count = self.db_req.get_images_in_file_count(item["file_id"])
                    file2_count = self.db_req.get_images_in_file_count(img["file_id"])

                    #self.log.debug("-- file 1 = %s, file 2 = %s" % (item["file_id"], img["file_id"],))
                    #self.log.debug("-- images count 1 = %s, images count 2 = %s" % (file1_count, file2_count,))

                    f2f_id = self.db_req.insert_file2file(
                        item["file_id"], img["file_id"], file1_count, file2_count,
                        )
                    self.db_req.inc_file2file( f2f_id, q_point)

                else:
                    if file2file["reverse"]:
                        q_point *= -1
                    #self.log.debug("-- inc count and add quality points")
                    self.db_req.inc_file2file( file2file["id"], q_point)
            else:
                # self.log.debug("Images compared: %s, quality points: %s" % (max(comp), q_point,))
                self.db_req.insert_image2image(
                    item["id"], img_id,
                    item["file_id"], img["file_id"],
                    max(comp), q_point
                )

                # self.log.debug("-- Next Item gray...")

        return

    def find_dupimage_bit_vectors(self, item):

        item_vs_bit = self.db_req.get_image_vectors_bitstr(item["id"])
        # self.log.debug("Item vector 10=%r" % item_v10)

        gray_ids = {}

        #self.log.debug("-- Get images gray like...")
        #t1 = datetime.now()

        imv = self.db_req.get_equal_images_bitstr(item_vs_bit["image_id"], item_vs_bit["bitmask"])

        for row in imv:
            gray_ids[ row['image_id'] ] = 1

        gray_ids = tuple(gray_ids.keys())
        gray_count = len(gray_ids)

        #t2 = datetime.now()
        #self.log.debug("-- Fetch images gray like: %s, in %s" % (gray_count, t2 - t1))

        item_vs_gray = None
        item_vs_cv = None
        # self.log.debug("Item vector 100=%r" % item_v100)

        if not gray_count:
            self.stats_req.inc("search_dupimages_processed", 1)

        for img_id in gray_ids:

            if self.stop_event.isSet():
                break

            self.stats_req.inc("search_dupimages_processed", 1.0 / gray_count)

            # self.log.debug("Item gray id=%r" % img_id)

            if img_id == item["id"]:
                #self.log.debug("-- Same image...")
                continue

            # self.log.debug("Item gray compare with item...")
            if self.db_req.has_image2image(item["id"], img_id):
                #self.log.debug("-- Already compared...")
                continue

            img = self.db_req.get_image(img_id)

            if not img:
                #self.log.debug("-- Image not found by id! Deleted?")
                continue

            if img["file_id"] == item["file_id"]:
                #self.log.debug("-- In the same file...")
                continue

            # self.log.debug("Item gray get vectors 100")
            img_vs_bit = self.db_req.get_image_vectors_bitstr(img_id)

            # self.log.debug("Item gray vectors 100=%r" % img_v100)

            sim = 0
            comp = 0

            if len(img_vs_bit) and len(item_vs_bit):

                vbig_1 = item_vs_bit["vector"]

                vbig_2 = img_vs_bit["vector"]

                comp = self.iMage.compare_bit_vectors(
                    vbig_1, vbig_2,
                )
                # self.log.debug("Item gray compared vs 100=%r" % (comp,))
                if comp >= self.max_diff:
                    sim += 1

            if sim:
                sim = 0

                # self.log.debug("Item gray get vectors 100")
                img_vs_gray = self.db_req.get_image_vectors_gray(img_id)

                if not item_vs_gray:
                    item_vs_gray = self.db_req.get_image_vectors_gray(item["id"])

                comp = 0

                if len(img_vs_gray) and len(item_vs_gray):

                    vbig_1 = item_vs_gray["vector"]

                    vbig_2 = img_vs_gray["vector"]

                    comp = self.iMage.compare_gray_vectors(
                        vbig_1, vbig_2,
                    )
                    # self.log.debug("Item gray compared vs 100=%r" % (comp,))
                    if comp >= self.max_diff:
                        sim += 1

            if sim:
                sim = 0

                # self.log.debug("Item gray get vectors 100")
                img_vs_cv = self.db_req.get_image_vector_numpy(img_id)

                if not item_vs_cv:
                    item_vs_cv = self.db_req.get_image_vector_numpy(item["id"])

                comp = 0

                if len(img_vs_cv) and len(item_vs_cv):

                    vbig_1 = item_vs_cv["vector"]

                    vbig_2 = img_vs_cv["vector"]

                    comp = self.iMage.compare_opencv_descriptors(
                        vbig_1, vbig_2,
                    )
                    # self.log.debug("Item gray compared vs 100=%r" % (comp,))
                    if comp >= self.max_diff:
                        sim += 1

            q_point = 0

            if sim:

                self.stats_req.inc("search_dupimages_similar", 1)

                #self.log.debug("-- found similar item=%r" % img)

                if item["mime"] != img["mime"]:
                    if item["mime"] == "image/png":
                        q_point += 2
                    if img["mime"] == "image/png":
                        q_point -= 2
                else:
                    if int(item["size"]) * 0.7 > int(img["size"]):
                        q_point += 1
                    if int(item["size"]) < int(img["size"]) * 0.7:
                        q_point -= 1

                square1 = int(item["width"]) * int(item["height"])
                square2 = int(img["width"]) * int(img["height"])
                if square1 > square2:
                    q_point += 2
                elif square1 < square2:
                    q_point -= 2

                if int(item["is_gray"]) > int(img["is_gray"]):
                    q_point -= 3
                else:
                    q_point += 3

                #self.log.debug("-- equal by %s, quality points: %s" % (max(comp), q_point,))
                self.db_req.insert_image2image(
                    item["id"], img_id,
                    item["file_id"], img["file_id"],
                    comp, q_point)

                file2file = self.db_req.has_file2file(item["file_id"], img["file_id"] )
                #self.log.debug("-- has File2File: %r" % (file2file,))
                if not file2file:

                    file1_count = self.db_req.get_images_in_file_count(item["file_id"])
                    file2_count = self.db_req.get_images_in_file_count(img["file_id"])

                    #self.log.debug("-- file 1 = %s, file 2 = %s" % (item["file_id"], img["file_id"],))
                    #self.log.debug("-- images count 1 = %s, images count 2 = %s" % (file1_count, file2_count,))

                    f2f_id = self.db_req.insert_file2file(
                        item["file_id"], img["file_id"], file1_count, file2_count,
                        )
                    self.db_req.inc_file2file( f2f_id, q_point)

                else:
                    if file2file["reverse"]:
                        q_point *= -1
                    #self.log.debug("-- inc count and add quality points")
                    self.db_req.inc_file2file( file2file["id"], q_point)
            else:
                # self.log.debug("Images compared: %s, quality points: %s" % (max(comp), q_point,))
                self.db_req.insert_image2image(
                    item["id"], img_id,
                    item["file_id"], img["file_id"],
                    comp, q_point
                )

                # self.log.debug("-- Next Item gray...")

        return

    def _worker(self):

        self.log.info("START: %r" % (self.server_address,))

        from service.task import TaskManager
        from service.statistics import StatisticsDirectRequest
        from project.query import Project

        task_manager = TaskManager(self.task_service_address)

        self.stats_req = task_manager.get_request().getStatistics()
        while not self.stats_req.ping():
            sleep(0.1)
        addr = self.stats_req.get_address()
        self.stats_req = StatisticsDirectRequest(addr)

        self.db_req = Project()

        self.min_diff = 1.0 - self.max_diff / self.max_vector

        from images.pil import iPIL
        self.iMage = iPIL()

        while not self.stop_event.isSet():

            try:
                data = self.queue.get_nowait()
            except Exception:
                data = None

            if data is None:
                sleep(0.1)
                continue

            try:
                self.find_dupimage_bit_vectors( data )

                self.stats_req.inc_imaging(1)
            except Exception as e:
                self.stats_req.inc("errors_count", 1)
                self.log.error("::_worker: %s" % (e,))
                self.log.debug("ERROR!\n%s" % (traceback.format_exc(),))
                pass

            self._lock.acquire()
            self.count_processed += 1
            self._lock.release()

            self.queue.task_done()


        self.log.info("STOP: %r" % (self.server_address,))

        self.stop_event.clear()
        return

    def process(self, image_row):
        self._lock.acquire()
        self.count_items += 1
        self._lock.release()

        self.queue.put_nowait(image_row)
        return True

    def reset_stats(self):
        self._lock.acquire()
        self.count_items = 0
        self.count_processed = 0
        self._lock.release()
        return True

    def empty(self):
        self._lock.acquire()
        is_empty = self.count_items == self.count_processed
        self._lock.release()
        return is_empty

    def get_stats(self):
        self._lock.acquire()
        item = {
            "items": self.count_items,
            "processed": self.count_processed
        }
        self._lock.release()
        return item

    def quit(self):
        self.log.info("quit event")

        import signal

        self.log.debug("send SIGINT to self")
        os.kill( os.getpid(), signal.SIGINT )

        return True

    def terminate_self(self, signum, frame):
        self.log.debug("got SHUTDOWN signal")
        t_begin = time()
        self.stop_event.set()
        self.stop_event.wait()
        self.server_close()
        t_end = time()
        self.log.info("SHUTDOWN in %s sec" % (timedelta(seconds=t_end - t_begin),))
        return

    pass


#---------------------------------------------

class SearchDupImageVectorRequest(object):

    manager = None

    def __init__(self, manager):
        self.manager = manager
        pass

    def process(self, number, image_row):
        return self.manager.sendmsg(number, "process", {"row": image_row})

    def reset_stats(self, number):
        return self.manager.sendmsg( number, "reset-stats", {})

    def empty(self, number):
        return self.manager.sendmsg( number, "empty", {})

    def get_stats(self, number):
        return self.manager.sendmsg( number, "get-stats", {})

    def ping(self, number):
        return self.manager.sendmsg( number, "ping", {}, 0.1)

    def ping_all(self):
        return self.manager.ping_all()

    def quit(self):
        return self.manager.quit_all()

    pass

#---------------------------------------------

class SearchDupImageVectorManager(ServiceManager):

    host = "127.0.1.11"
    port_min = 10000
    port_max = 20000
    ports = None
    request = None

    task_service_address = None

    log = None

    def __init__(self, task_service_address):
        self.task_service_address = task_service_address
        self.ports = {}
        self.log = get_file_logger("SearchDupImageVectorManager")
        pass

    def start_service(self, number):

        number = str(number)

        runing = False
        if number not in self.ports:

            port = random.randint(self.port_min, self.port_max)

            self.ports[number] = port

            address = (self.host, port,)

            Popen([self.getPython(), "service/search_dupimage_vector.py",
                   "--host", "%s" % self.host,
                   "--port", "%s" % port,
                   "--task-service", "%s:%s" % self.task_service_address
            ])

            # WAIT WHILE SERVICE STARTS

            pings = 100
            t_begin = time()
            while not runing:

                pong = self.get_request().ping(number)

                if pong == "pong":
                    runing = True

                pings -= 1
                if pings <= 0:
                    break

            if pings <= 0:
                del self.ports[number]
                self.log.error("FAIL TO START service(%s) on %r in %s sec" % (
                    number,
                    address, timedelta(seconds=time() - t_begin),))
            else:
                self.log.info("STARTED service(%s) on %r in %s sec" % (
                    number,
                    address, timedelta(seconds=time() - t_begin),))

        return runing

    def sendmsg(self, number, task, data, timeout=default_request_timeout):
        number = str(number)
        port = self.ports[number]
        address = (self.host, port,)
        try:
            result = sendmsg( address, {"task":task, "data":data}, timeout=timeout )
        except Exception as e:
            result = None
            self.log.error("SOCKET: %s" % ( e,))
            self.log.debug("FAIL SEND MESSAGE: task=%s, data=%r" % (
                task, data,))
        return result

    def quit_all(self):
        result = True
        ports = self.ports.copy()
        for number, port in ports.items():
            ret = self.sendmsg(number, "quit", {}, 0.1)
            if ret is None:
                result = False
            else:
                del self.ports[ number ]
        return result

    def ping_all(self):
        result = False
        ports = self.ports.copy()
        for number, port in ports.items():
            ret = self.get_request().ping(number)
            if ret == "pong":
                result = True
        return result

    def get_request(self):
        if not self.request:
            self.request = SearchDupImageVectorRequest(self)
        return self.request

    pass


#---------------------------------------------

def run_service(host, port, task_service_address):

    if hasattr(os, "nice"):
        os.nice(default_niceness_level)

    server = None
    e_code = 0
    try:
        SERVER_ADDRESS = (host, int(port),)

        server = SearchDupImageVectorServer(SERVER_ADDRESS, SearchDupImageVectorHandler, bind_and_activate=False)
        if task_service_address:
            host, port = task_service_address.split(':')
            server.task_service_address = (host, int(port),)

        import signal

        signal.signal(signal.SIGINT, server.terminate_self)

        server.server_bind()
        server.server_activate()
        server.serve_forever()
    except Exception as e:
        e_code = 1
        if server and server.log:
            server.log.error("run_service: ERROR=%r" % (e,))
        pass

    return e_code


#---------------------------------------------

def launch():

    from argparse import ArgumentParser

    parser = ArgumentParser(description='Run search for duplicate image vector processing service.')
    parser.add_argument('--host', dest='host', action='store', type=str,
        help='Listen on TCP address')
    parser.add_argument('--port', dest='port', action='store', type=int,
        help='Listen on TCP port')
    parser.add_argument('--task-service', dest='task_service', action='store', type=str,
        help='Task service listen on TCP address')

    args = parser.parse_args()
    return run_service(args.host, args.port, args.task_service)

if __name__ == "__main__":
    sys.exit(launch())
